package com.chance.cc.crawler.development.scripts.tianmao;

import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.tags.CategoryTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

/**
 * @author bx
 * @date 2020/11/17 0017 17:18
 */
public class TianmaoSearchHtmlCrawlerScript extends CrawlerCommonScript {

    private static final Logger logger = LoggerFactory.getLogger(TianmaoSearchHtmlCrawlerScript.class);

    private static final String searchUrl = "https://s.taobao.com/search?q=%s&js=1" +
            "&stats_click=search_radio_tmall%%3A1&tab=mall&ie=utf8&p4ppushleft=%%2C44&sort=default&s=%d";

    private static final String searchFirstUrl = "https://s.taobao.com/search?q=%s&js=1" +
            "&stats_click=search_radio_tmall%%3A1&tab=mall&ie=utf8&p4ppushleft=%%2C44&sort=default&s=0";

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        try {
            Thread.sleep(RandomUtils.nextInt(3,6)*1000L);
        } catch (InterruptedException e) {
            logger.error(e.getMessage(),e);
        }
        List<CrawlerRequestRecord> crawlerRequestRecords = new ArrayList<>();

        String requestUrl = page.getRequest().getUrl();
        List<NameValuePair> parse = URLEncodedUtils.parse(requestUrl.split("\\?")[1], Charset.forName("utf-8"));
        int s = 0;
        String keyWord = "";
        for (NameValuePair nameValuePair : parse) {
            switch (nameValuePair.getName()){
                case "s": s = Integer.valueOf(nameValuePair.getValue()); break;
                case "q": keyWord = nameValuePair.getValue(); break;
            }
        }
        Json dataJson;
        try {
            String dataScript = extractDataInfo(page);
            if (StringUtils.isBlank(dataScript)){
                throw new Exception("extract script information to report an error！");
            }
            dataJson = new Json(dataScript);
            dataJson.jsonPath($_type + ".mods.itemlist.data.auctions");
        } catch (Exception e) {
            logger.error("taobao product comment download failed,error message {},page rawText detail {}",
                    e.getMessage(),page.getRawText());
            try {
                Thread.sleep(RandomUtils.nextInt(5,10)*1000L);
            } catch (InterruptedException e1) {
                logger.error(e1.getMessage(),e1);
            }
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
            crawlerRecord.setNeedWashPage(false); //下载异常，不需要清洗页面
            return crawlerRequestRecords;
        }

        if (StringUtils.isBlank(keyWord)){
            logger.error("taobao search keyword is null,request url:" + requestUrl);
            crawlerRecord.setNeedWashPage(false);
            return null;
        }

        String nextPageurl = null;
        try {
            keyWord = URLEncoder.encode(keyWord,"utf-8");
            int totalPage = Integer.valueOf(dataJson.jsonPath($_type + ".mods.sortbar.data.pager.totalPage").get());
            int currentPage = Integer.valueOf(dataJson.jsonPath($_type + ".mods.sortbar.data.pager.currentPage").get());
            if (currentPage < totalPage){ // 100页
                nextPageurl = String.format(searchUrl,keyWord,(s + 44));
            } else {
                logger.info("taobao search html keyword {},turn page finish,total page {}", URLDecoder.decode(keyWord,"utf-8"),totalPage);
            }
            if (StringUtils.isNotBlank(nextPageurl)){
                CrawlerRequestRecord nextCrawlerRequestRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRecord)
                        .httpUrl(nextPageurl)
                        .releaseTime(System.currentTimeMillis())
                        .needWashed(true)
                        .httpHeads(page.getRequest().getHeaders())
                        .build();

                crawlerRequestRecords.add(nextCrawlerRequestRecord);
            }
        } catch (UnsupportedEncodingException e) {
            logger.error(e.getMessage(),e);
        }
        return crawlerRequestRecords;
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {

        List<CrawlerData> crawlerDataList = new ArrayList<>();

        Json dataJson = new Json(extractDataInfo(page));
        String keyword = dataJson.jsonPath($_type + ".mods.p4p.data.p4pconfig.keyword").get();

        try {
            keyword = URLDecoder.decode(keyword,"utf-8");
        } catch (UnsupportedEncodingException e) {
            logger.error(e.getMessage(),e);
        }

        List<String> itemList = dataJson.jsonPath($_type + ".mods.itemlist.data.auctions").all();
        for (String item : itemList) {
            Json itemJson = new Json(item);
            String itemId = itemJson.jsonPath($_type + ".nid").get();
            String sellerId = itemJson.jsonPath($_type + ".user_id").get();
            String detailUrl = itemJson.jsonPath($_type + ".detail_url").get();
            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRecord,page)
                    .dataId(StringUtils.joinWith("-",crawlerRecord.getDomain(),keyword,CrawlerEnum.CrawlerDataType.article,itemId,sellerId))
                    .releaseTime(System.currentTimeMillis())
                    .addContentKV("dataId", StringUtils.joinWith("-",crawlerRecord.getDomain(),itemId,sellerId))
                    .addContentKV("itemId", itemId)
                    .addContentKV("userId", sellerId)
                    .addContentKV("detailUrl", "https:"+detailUrl)
                    .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                    .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                    .build();
            crawlerDataList.add(crawlerData);
        }

        return crawlerDataList;
    }

    @Override
    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord requestRecord, List<CrawlerRequestRecord> supportSourceRecords) {

        List<CrawlerRecord> allItemRecords = new ArrayList<>();

        if (supportSourceRecords == null || supportSourceRecords.size() <1){
            return super.prepareRequest(requestRecord,supportSourceRecords);
        }

        CrawlerRequestRecord keywordRecord = supportSourceRecords.get(0);
        HttpPage internalDownloadPage = keywordRecord.getInternalDownloadPage();
        try {
            Json keywordJson = new Json(internalDownloadPage.getRawText());
            String status = keywordJson.jsonPath($_type + ".status").get();
            List<String> contents = keywordJson.jsonPath($_type + ".content").all();
            if ("0".equals(status) && contents != null && contents.size() > 0){
                Json contentJson = new Json(contents.get(0));
                String keys = contentJson.jsonPath($_type + ".keys").get();
                List<String> keyList = new Json(keys).toList(String.class);
                for (String key : keyList) {
                    String keyword = URLEncoder.encode(key, "utf-8");
                    CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                            .turnPageRequest(requestRecord)
                            .httpUrl(String.format(searchFirstUrl,keyword))
                            .httpHead("referer","https://s.taobao.com/")
                            .httpHead("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36")
                            .httpHead("cookie","_samesite_flag_=true; cookie2=1b657d46efd4104d1dfe089dc4a1835c; t=6952f998ef3b907ad51292a7cbf8cb4d; _tb_token_=335abab33b9ea; cna=EQk9GPo4tywCATohFCIv0irN; xlly_s=1; lgc=%5Cu7CFB%5Cu8FD8%5Cu7B97%5Cu4E0D; dnk=%5Cu7CFB%5Cu8FD8%5Cu7B97%5Cu4E0D; tracknick=%5Cu7CFB%5Cu8FD8%5Cu7B97%5Cu4E0D; _cc_=UIHiLt3xSw%3D%3D; sgcookie=E1002dnfUSdAdGIP6yADLAzWStxjhLkgB0F12nqltpl1wQb5WvmUK9C%2FBqBYFnfvuCpaeJb9e59jRdwPZWyrAKQu7A%3D%3D; uc3=nk2=rL9CGfhTA0w%3D&lg2=UIHiLt3xD8xYTw%3D%3D&vt3=F8dCufwsARmN2xd%2B3hg%3D&id2=UUkJYAV8aFQ8rg%3D%3D; csg=72fcf25b; skt=feb5b85bc8e9665d; existShop=MTYwNTc3MDE4Mw%3D%3D; uc4=nk4=0%40rpHMnVi0ocDUvwAd92lMKLQEmg%3D%3D&id4=0%40U2uFVlzHXCqDYyRTGjsxkgJMFzGo; mt=ci=0_1; thw=cn; v=0; _m_h5_tk=77c878ea553015b36a760c4109aab104_1605777392010; _m_h5_tk_enc=0239c12254ea1dfbcf9f98972b119114; enc=oIjV9uj7Tq8%2B9NhSiPMLibYaG9mOrHlUetl6NApTTB9pnppJIqD6ED3KAGsPPEdkdbBl9nFihtJJowHXgm7n4w%3D%3D; alitrackid=www.taobao.com; lastalitrackid=www.taobao.com; hng=CN%7Czh-CN%7CCNY%7C156; x5sec=7b227365617263686170703b32223a226266353434363536356131363639623737396332343132353064653937643437434f336132503046454a62653959727171764f564f526f4d4d6a45334e6a41304e7a41304f447378227d; uc1=cookie21=UIHiLt3xThH8t7YQp6aL&cookie14=Uoe0aDvbLb%2FDQQ%3D%3D&cookie16=U%2BGCWk%2F74Mx5tgzv3dWpnhjPaQ%3D%3D&pas=0&existShop=false; JSESSIONID=E9FB2FED57291EFF4ADD0698C29D767C; isg=BNvb7dB4ANT3MXygptabS1kxaj9FsO-ytnpZP80YVVqDrPmOVIOZA_BqRgwii0eq; tfstk=cMrOB0cyR6fi7OHLUPQHc7f8HZZOaUQti6cyDT_c85gMR1tvcsc8qudScHDJ6DBd.; l=eBOxt66gO3-sjI3vBO5Cnurza779fIRbz1PzaNbMiInca69cTnae3NQVzOF97dtjgt5cLeKr09hQ7RH67k438xgKqelyRs5mpZJ68e1..")
                            .releaseTime(System.currentTimeMillis())
                            .needWashed(true)
                            .build();
                    allItemRecords.add(crawlerRequestRecord);
                }
            }
        } catch (Exception e) {
            logger.error(e.getMessage(),e);
        }

        return allItemRecords;
    }

    @Override
    public String domain() {
        return "tianmao";
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular("https://s.taobao.com/search\\S*");
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        return true;
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }

    private void addCrawlerRecords(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord){

        Map<String, CategoryTag> tags = crawlerRecord.getTags();
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 1;
        if (crawlerBusinessTags.hasKVTag("taobao_download_retry_count")){
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag("taobao_download_retry_count").getVal();
            if (count >= 5){
                logger.error("taobao search keyword download he number of retries exceeds the limit" +
                        ",request url {},download detail {}",crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }
        count++;
        crawlerBusinessTags.addCustomKV("taobao_download_retry_count",count);

        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .turnPageRequest(crawlerRecord)
                .httpUrl(crawlerRecord.getHttpRequest().getUrl())
                .needWashed(true)
                .releaseTime(System.currentTimeMillis())
                .build();
        crawlerRequestRecord.setTags(tags);
        crawlerRequestRecords.add(crawlerRequestRecord);
    }

    private String extractDataInfo(HttpPage page){
        List<String> scripts = page.getHtml().xpath("//script").all();
        String dataScript = null;
        for (String script : scripts) {
            script = script.trim();
            String scriptPrefix = "g_page_config = ";
            if (script.startsWith(scriptPrefix)){
                String[] scriptArray = script.split("\n");
                for (String sc : scriptArray) {
                    if (sc.trim().startsWith(scriptPrefix)){
                        dataScript = sc.substring(scriptPrefix.length(),sc.lastIndexOf(";"));
                        break;
                    }
                }
                break;
            }
        }

        return dataScript;
    }

}
